# -*- coding: utf-8 -*-
"""
Created on Fri Dec 04 10:30:07 2015

@author: liliangrong
"""
#抓取页面源代码
def get_page_code(words):
    #import urllib
    import urllib2
    #import re
    user_agent = "Mozilla/5.0 (Windows NT 5.1; rv:35.0) Gecko/20100101 Firefox/35.0"
    headers = {"User-Agent":user_agent}
    url = "http://www.baidu.com/baidu?wd=%s&tn=monline_4_dg" % words
    try:
        req = urllib2.Request(url,None,headers)
        resp = urllib2.urlopen(req)
        page_source = resp.read()
        return page_source

    except urllib2.URLError,e:
        if hasattr(e,'code'):
            print e.code
        if hasattr(e,'reason'):
            print e.reason
def get_sort_url(words):
    import re
    page_source = get_page_code(words)
    #抓取排名序号和对应的网址前缀
    pattern = re.compile('<div .*?id="([\d]+)" class="result c-container ">.*?<span class="c-showurl">(.*?)</span>.*?</div>',re.S)
    #返回正则表达式处理后的内容
    sort_url = re.findall(pattern,page_source)
    return sort_url

def get_page_num(words):
    import re
    page_source = get_page_code(words)
    pattern = re.compile('<div id="page">.*?<a href="(.*?)">.*?<span class="pc">([\d])</span></a>.*?</div>',re.S)
    page_num = re.findall(pattern,page_source)
    return page_num
    
def get_baidu_rank(site_url,):
    pass
    #sort_urls = get_sort_url(words)
    
if __name__ == "__main__":
    import time
    start = time.clock()
    word = raw_input("input word : ")
    site_url = raw_input("input site_url : ")
    f = open('baidu_rank.txt','w')
    
    for pn in xrange(1,10):
        url_lists = get_url_list(pn)
        for item in url_lists:
            if item[0].strip() != None and item[1].strip() != None:
                line = item[1]+" : "+item[0]
                f.write(line)
                f.write("\n")
                #f.write("\n----------------------------------------------------\n")
    f.close()
        
    end = time.clock()
    print "run time is %f s" % (end-start)